from typing import Iterator, AsyncIterator
from langchain_core.document_loaders import BaseLoader
from langchain_core.documents import Document


class CustomDocumentLoader(BaseLoader):
    def __init__(self,file_path:str)->None:
        self.file_path = file_path

    def lazy_load(self)->Iterator[Document]:
        with open(self.file_path,encoding="utf-8") as f:
            line_number = 0
            for line in f:
                yield Document(page_content=line,metadata={"line_number":line_number,"source":self.file_path})
                line_number += 1

    async def alazy_load(self) -> AsyncIterator[Document]:
        import aiofiles
        async with aiofiles.open(self.file_path,encoding="utf-8") as f:
            line_number = 0
            async for line in f:
                yield Document(page_content=line,metadata={"line_number":line_number,"source":self.file_path})
                line_number += 1

loader = CustomDocumentLoader("./eshop_goods.txt")
documents =loader.load()

print(documents)
print(len(documents))
print(documents[0].metadata)